| import re | |
| import numpy as np | |
| import psutil | |
| import os | |
| from tqdm.auto import tqdm | |
| import logging | |
| logger = logging.getLogger(__name__) | |
| def get_current_ram_usage(): | |
| ram = psutil.virtual_memory() | |
| return ram.available / 1024 / 1024 / 1024, ram.total / 1024 / 1024 / 1024 | |
| def download_models(models): | |
| for model in tqdm(models, desc="Downloading models"): | |
| logger.info(f"Downloading {model}") | |
| for i in range(0, 5): | |
| curr_dir = f"{model}/train_{i}/best_model/" | |
| os.makedirs(curr_dir, exist_ok=True) | |
| os.system( | |
| f"wget -q https://huggingface.co/researchaccount/{model}/resolve/main/train_{i}/best_model/config.json -P {curr_dir}" | |
| ) | |
| os.system( | |
| f"wget -q https://huggingface.co/researchaccount/{model}/resolve/main/train_{i}/best_model/pytorch_model.bin -P {curr_dir}" | |
| ) | |
| os.system( | |
| f"wget -q https://huggingface.co/researchaccount/{model}/resolve/main/train_{i}/best_model/special_tokens_map.json -P {curr_dir}" | |
| ) | |
| os.system( | |
| f"wget -q https://huggingface.co/researchaccount/{model}/resolve/main/train_{i}/best_model/tokenizer_config.json -P {curr_dir}" | |
| ) | |
| os.system( | |
| f"wget -q https://huggingface.co/researchaccount/{model}/resolve/main/train_{i}/best_model/training_args.bin -P {curr_dir}" | |
| ) | |
| os.system( | |
| f"wget -q https://huggingface.co/researchaccount/{model}/resolve/main/train_{i}/best_model/vocab.txt -P {curr_dir}" | |
| ) | |
| def softmax(x): | |
| return np.exp(x) / sum(np.exp(x)) | |
| def ga(file): | |
| code = """ | |
| <!-- Global site tag (gtag.js) - Google Analytics --> | |
| <script async src="https://www.googletagmanager.com/gtag/js?id=G-NH9HWCW08F"></script> | |
| <script> | |
| window.dataLayer = window.dataLayer || []; | |
| function gtag(){dataLayer.push(arguments);} | |
| gtag('js', new Date()); | |
| gtag('config', 'G-NH9HWCW08F'); | |
| </script> | |
| """ | |
| a = os.path.dirname(file) + "/static/index.html" | |
| with open(a, "r") as f: | |
| data = f.read() | |
| if len(re.findall("G-", data)) == 0: | |
| with open(a, "w") as ff: | |
| newdata = re.sub("<head>", "<head>" + code, data) | |
| ff.write(newdata) | |