File size: 707 Bytes
dd74f4c |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 |
import os
import sys
import subprocess
print("Getting rustup")
subprocess.run(
"curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y",
shell=True,
)
print("Got rustup")
myenv = os.environ.copy()
myenv["PATH"] = os.path.expanduser("~/.cargo/bin:") + myenv["PATH"]
print("RUSTC", os.path.isfile(os.path.expanduser("~/.cargo/bin/rustc")))
subprocess.run("rustc --version", shell=True, env=myenv)
subprocess.run(
"pip install -e git+https://github.com/huggingface/tokenizers/#egg=tokenizers\&subdirectory=bindings/python",
shell=True,
env=myenv,
)
sys.path.append(
os.path.join(os.getcwd(), "src", "tokenizers", "bindings", "python", "py_src")
)
import tokenizers
|