|
|
--- |
|
|
library_name: transformers |
|
|
tags: [] |
|
|
--- |
|
|
|
|
|
# Tokenizer Card |
|
|
|
|
|
<!-- Provide a quick summary of what the model is/does. --> |
|
|
|
|
|
```bash |
|
|
from transformers import AutoTokenizer |
|
|
|
|
|
tokenizer = AutoTokenizer.from_pretrained("azherali/python-code-search-tokenizer") |
|
|
|
|
|
example = """class LinearLayer(): |
|
|
def __init__(self, input_size, output_size): |
|
|
self.weight = torch.randn(input_size, output_size) |
|
|
self.bias = torch.zeros(output_size) |
|
|
|
|
|
def __call__(self, x): |
|
|
return x @ self.weights + self.bias |
|
|
""" |
|
|
tokenizer.tokenize(example) |
|
|
|
|
|
|
|
|
|
|
|
|
