small improvement by training with part of data converted to lower case

config.json

@@ -33,6 +33,9 @@
   "num_hidden_layers": 12,
   "output_past": true,
   "pad_token_id": 1,
+  "position_embedding_type": "absolute",
+  "transformers_version": "4.3.2",
   "type_vocab_size": 1,
+  "use_cache": true,
   "vocab_size": 32005
 }

pytorch_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:72ac83b9b1ded370c85082c67735f7ef250a333aa37e6b3434b0dc0b210b1988
-size 440227047
+oid sha256:55e4e2730fadcb7f4905c10d6f39abc0fe59ead7f919e5fde845f39144cc430b
+size 440227439