Rotary Scaling Factor of 4 for 8k context (Do not merge)

config.json

@@ -38,7 +38,7 @@
   "rotary_emb_fraction": 1.0,
   "rotary_emb_interleaved": false,
   "rotary_emb_scale_base": null,
-  "rotary_scaling_factor": null,
+  "rotary_scaling_factor": 4.0,
   "scale_attn_by_inverse_layer_idx": false,
   "scale_attn_weights": true,
   "summary_activation": null,