Repository: openlawlibrary/pygls

Issue:
`exclude_none = True` breaks `shutdown` requests 
Unfortunately, it looks like #236 breaks [`shutdown`](https://microsoft.github.io/language-server-protocol/specifications/lsp/3.17/specification/#shutdown) requests.

I have a [command](https://github.com/swyddfa/esbonio/blob/ea8311a2b79ab8ef74157e9d5b2635e6b7b6a9e9/code/src/lsp/client.ts#L256) in Esbonio's VSCode extension that restarts the language server which sends a `shutdown` request to the server resulting in the following error

```
Command 'Esbonio: Restart Language Server' resulted in an error 
(The received response has neither a result or an error property.)
```

With VSCode's devtools open I can see the response received from the server and the `result` field is in fact missing.
```json
{
    "jsonrpc": "2.0",
    "id": 1
}
```
Changing `exclude_none` back to `exclude_unset` fixes this issue.

I'm not entirely sure what to suggest as a fix as while I didn't follow the discussion around #236 that closely I'm aware it was to resolve some other issues.

---
Edit: Actually it looks like this is an issue for any request where `"result": null` is a valid response which includes completions, goto definition and others

Discussion:
tombh: Amazing. It looks great to me. Let's merge whenever you're ready.
alcarney: It's ready to go, I guess we just need a formal review to enable the merge button?

Patch:
diff --git a/pygls/lsp/types/basic_structures.py b/pygls/lsp/types/basic_structures.py
index 879c15b..2b28b59 100644
--- a/pygls/lsp/types/basic_structures.py
+++ b/pygls/lsp/types/basic_structures.py
@@ -54,6 +54,14 @@ class Model(BaseModel):
             'from_': 'from'
         }
 
+    def __init__(self, **data: Any) -> None:
+        super().__init__(**data)
+
+        # Serialize (.json()) fields that has default value which is not None
+        for name, field in self.__fields__.items():
+            if getattr(field, 'default', None) is not None:
+                self.__fields_set__.add(name)
+
 
 class JsonRpcMessage(Model):
     """A base json rpc message defined by LSP."""
diff --git a/pygls/protocol.py b/pygls/protocol.py
index 7e03b7d..a5438cc 100644
--- a/pygls/protocol.py
+++ b/pygls/protocol.py
@@ -385,7 +385,7 @@ class JsonRPCProtocol(asyncio.Protocol):
             return
 
         try:
-            body = data.json(by_alias=True, exclude_none=True, encoder=default_serializer)
+            body = data.json(by_alias=True, exclude_unset=True, encoder=default_serializer)
             logger.info('Sending data: %s', body)
 
             body = body.encode(self.CHARSET)


Test patch:
diff --git a/tests/test_protocol.py b/tests/test_protocol.py
index 88e0a2d..5318b80 100644
--- a/tests/test_protocol.py
+++ b/tests/test_protocol.py
@@ -14,6 +14,7 @@
 # See the License for the specific language governing permissions and      #
 # limitations under the License.                                           #
 ############################################################################
+import io
 import json
 from concurrent.futures import Future
 from functools import partial
@@ -25,8 +26,16 @@ import pytest
 
 from pygls.exceptions import JsonRpcException, JsonRpcInvalidParams
 from pygls.lsp import Model, get_method_params_type
-from pygls.lsp.types import ClientCapabilities, InitializeParams, InitializeResult
-from pygls.protocol import JsonRPCNotification, JsonRPCRequestMessage, JsonRPCResponseMessage
+from pygls.lsp.types import (
+    ClientCapabilities,
+    CompletionItem,
+    CompletionItemKind,
+    InitializeParams,
+    InitializeResult,
+    ProgressParams,
+    WorkDoneProgressBegin,
+)
+from pygls.protocol import JsonRPCNotification, JsonRPCProtocol, JsonRPCRequestMessage, JsonRPCResponseMessage
 from pygls.protocol import deserialize_message as _deserialize_message
 
 TEST_METHOD = 'test_method'
@@ -94,6 +103,47 @@ def test_deserialize_notification_message_bad_params_should_raise_error():
         json.loads(params, object_hook=deserialize_message)
 
 
+@pytest.mark.parametrize(
+    "params, expected",
+    [
+        (
+            ProgressParams(
+                token="id1",
+                value=WorkDoneProgressBegin(
+                    title="Begin progress",
+                    percentage=0,
+                )
+            ),
+            {
+                "jsonrpc": "2.0",
+                "method": "test/notification",
+                "params": {
+                    "token": "id1",
+                    "value": {
+                        "kind": "begin",
+                        "percentage": 0,
+                        "title": "Begin progress"
+                    }
+                }
+            }
+        ),
+    ]
+)
+def test_serialize_notification_message(params, expected):
+    """Ensure that we can serialize notification messages, retaining all expected fields."""
+
+    buffer = io.StringIO()
+
+    protocol = JsonRPCProtocol(None)
+    protocol._send_only_body = True
+    protocol.connection_made(buffer)
+
+    protocol.notify("test/notification", params=params)
+    actual = json.loads(buffer.getvalue())
+
+    assert actual == expected
+
+
 def test_deserialize_response_message():
     params = '''
     {
@@ -110,7 +160,6 @@ def test_deserialize_response_message():
     assert result.result == "1"
     assert result.error is None
 
-
 def test_deserialize_request_message_with_registered_type__should_return_instance():
     params = '''
     {
@@ -163,6 +212,51 @@ def test_deserialize_request_message_without_registered_type__should_return_name
     assert result.params.field_b.inner_field == "test_inner"
 
 
+@pytest.mark.parametrize(
+    "result, expected",
+    [
+        (None, {"jsonrpc": "2.0", "id": "1", "result": None}),
+        (
+            [
+                CompletionItem(label='example-one'),
+                CompletionItem(
+                    label='example-two',
+                    kind=CompletionItemKind.Class,
+                    preselect=False,
+                    deprecated=True
+                ),
+            ],
+            {
+                "jsonrpc": "2.0",
+                "id": "1",
+                "result": [
+                    {"label": "example-one"},
+                    {
+                        "label": "example-two",
+                        "kind": 7, # CompletionItemKind.Class
+                        "preselect": False,
+                        "deprecated": True
+                    }
+                ]
+            }
+        ),
+    ]
+)
+def test_serialize_response_message(result, expected):
+    """Ensure that we can serialize response messages, retaining all expected fields."""
+
+    buffer = io.StringIO()
+
+    protocol = JsonRPCProtocol(None)
+    protocol._send_only_body = True
+    protocol.connection_made(buffer)
+
+    protocol._send_response("1", result=result)
+    actual = json.loads(buffer.getvalue())
+
+    assert actual == expected
+
+
 def test_data_received_without_content_type_should_handle_message(client_server):
     _, server = client_server
     body = json.dumps({
@@ -278,4 +372,3 @@ def test_ignore_unknown_notification(client_server):
 
     # Remove mock
     server.lsp._execute_notification = fn
-