| { | |
| "max_length": 512, | |
| "context_candidates": [ | |
| null, | |
| 500, | |
| 300, | |
| 150, | |
| 80, | |
| 30, | |
| 0 | |
| ], | |
| "marker_tokens": [ | |
| "[ENT]", | |
| "[/ENT]" | |
| ], | |
| "marker_token_ids": { | |
| "[ENT]": 65000, | |
| "[/ENT]": 65001 | |
| }, | |
| "splits": { | |
| "train": { | |
| "rows": 100796, | |
| "token_length_min": 5, | |
| "token_length_mean": 53.74587285209731, | |
| "token_length_median": 31.0, | |
| "token_length_p95": 174.0, | |
| "token_length_p99": 393.25000000001455, | |
| "token_length_max": 512, | |
| "context_counts": { | |
| "None": 100590, | |
| "500": 206 | |
| } | |
| }, | |
| "validation": { | |
| "rows": 12600, | |
| "token_length_min": 5, | |
| "token_length_mean": 52.443174603174604, | |
| "token_length_median": 31.0, | |
| "token_length_p95": 165.0, | |
| "token_length_p99": 393.0, | |
| "token_length_max": 512, | |
| "context_counts": { | |
| "None": 12570, | |
| "500": 30 | |
| } | |
| }, | |
| "test": { | |
| "rows": 12600, | |
| "token_length_min": 5, | |
| "token_length_mean": 53.945238095238096, | |
| "token_length_median": 31.0, | |
| "token_length_p95": 178.0, | |
| "token_length_p99": 409.0, | |
| "token_length_max": 512, | |
| "context_counts": { | |
| "None": 12575, | |
| "500": 25 | |
| } | |
| } | |
| } | |
| } |