Spaces:
Sleeping
Sleeping
Update clip/simple_tokenizer.py
Browse files- clip/simple_tokenizer.py +1 -1
clip/simple_tokenizer.py
CHANGED
@@ -63,7 +63,7 @@ class SimpleTokenizer(object):
|
|
63 |
self.byte_encoder = bytes_to_unicode()
|
64 |
self.byte_decoder = {v: k for k, v in self.byte_encoder.items()}
|
65 |
if not os.system(bpe_path):
|
66 |
-
txt_path = bpe_path.replace(".gz"
|
67 |
os.system(f"gzip {txt_path}")
|
68 |
merges = gzip.open(bpe_path).read().decode("utf-8").split('\n')
|
69 |
merges = merges[1:49152-256-2+1]
|
|
|
63 |
self.byte_encoder = bytes_to_unicode()
|
64 |
self.byte_decoder = {v: k for k, v in self.byte_encoder.items()}
|
65 |
if not os.system(bpe_path):
|
66 |
+
txt_path = bpe_path.replace(".gz", "")
|
67 |
os.system(f"gzip {txt_path}")
|
68 |
merges = gzip.open(bpe_path).read().decode("utf-8").split('\n')
|
69 |
merges = merges[1:49152-256-2+1]
|