remove embedded hugging face tokens
Browse files- hf_benchmark.py +5 -2
- paper_baseline_check.py +5 -2
hf_benchmark.py
CHANGED
|
@@ -24,7 +24,7 @@ import csv
|
|
| 24 |
from pathlib import Path
|
| 25 |
|
| 26 |
# ββ HuggingFace token βββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 27 |
-
HF_TOKEN = os.environ.get("HF_TOKEN"
|
| 28 |
|
| 29 |
# ββ ArgΓΌmanlar ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 30 |
parser = argparse.ArgumentParser()
|
|
@@ -38,7 +38,10 @@ OUT.mkdir(parents=True, exist_ok=True)
|
|
| 38 |
|
| 39 |
# ββ HuggingFace login βββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 40 |
from huggingface_hub import login
|
| 41 |
-
|
|
|
|
|
|
|
|
|
|
| 42 |
print("HuggingFace login β")
|
| 43 |
|
| 44 |
# ββ Tokenizer yΓΌkle βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
|
|
|
| 24 |
from pathlib import Path
|
| 25 |
|
| 26 |
# ββ HuggingFace token βββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 27 |
+
HF_TOKEN = os.environ.get("HF_TOKEN")
|
| 28 |
|
| 29 |
# ββ ArgΓΌmanlar ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 30 |
parser = argparse.ArgumentParser()
|
|
|
|
| 38 |
|
| 39 |
# ββ HuggingFace login βββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 40 |
from huggingface_hub import login
|
| 41 |
+
if HF_TOKEN:
|
| 42 |
+
login(token=HF_TOKEN, add_to_git_credential=False)
|
| 43 |
+
else:
|
| 44 |
+
print("HF_TOKEN not set; using existing Hugging Face login state if available.")
|
| 45 |
print("HuggingFace login β")
|
| 46 |
|
| 47 |
# ββ Tokenizer yΓΌkle βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
paper_baseline_check.py
CHANGED
|
@@ -14,8 +14,11 @@ from huggingface_hub import login
|
|
| 14 |
from datasets import load_dataset
|
| 15 |
from turkish_tokenizer import TurkishTokenizer
|
| 16 |
|
| 17 |
-
HF_TOKEN = os.environ.get("HF_TOKEN"
|
| 18 |
-
|
|
|
|
|
|
|
|
|
|
| 19 |
|
| 20 |
base = TurkishTokenizer()
|
| 21 |
|
|
|
|
| 14 |
from datasets import load_dataset
|
| 15 |
from turkish_tokenizer import TurkishTokenizer
|
| 16 |
|
| 17 |
+
HF_TOKEN = os.environ.get("HF_TOKEN")
|
| 18 |
+
if HF_TOKEN:
|
| 19 |
+
login(token=HF_TOKEN, add_to_git_credential=False)
|
| 20 |
+
else:
|
| 21 |
+
print("HF_TOKEN not set; using existing Hugging Face login state if available.")
|
| 22 |
|
| 23 |
base = TurkishTokenizer()
|
| 24 |
|