File size: 2,344 Bytes
90afd57
3831b68
854b7af
412c787
5ede0fb
d44f5d8
 
 
854b7af
 
 
 
 
0558cbb
 
 
5ede0fb
d44f5d8
5d082b0
 
2de650e
5d082b0
a49ffd4
5d082b0
 
a49ffd4
5d082b0
 
a49ffd4
5d082b0
 
a49ffd4
5d082b0
 
a49ffd4
5d082b0
 
a49ffd4
5d082b0
3831b68
 
 
 
90afd57
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
import re
import numpy as np
import psutil
import os
from tqdm.auto import tqdm
import logging

logger = logging.getLogger(__name__)


def get_current_ram_usage():
    ram = psutil.virtual_memory()
    return ram.available / 1024 / 1024 / 1024, ram.total / 1024 / 1024 / 1024


def download_models(models):
    for model in tqdm(models, desc="Downloading models"):
        logger.info(f"Downloading {model}")
        for i in range(0, 5):
            curr_dir = f"{model}/train_{i}/best_model/"
            os.makedirs(curr_dir, exist_ok=True)
            os.system(
                f"wget -q https://huggingface.co/researchaccount/{model}/resolve/main/train_{i}/best_model/config.json -P {curr_dir}"
            )
            os.system(
                f"wget -q https://huggingface.co/researchaccount/{model}/resolve/main/train_{i}/best_model/pytorch_model.bin -P {curr_dir}"
            )
            os.system(
                f"wget -q https://huggingface.co/researchaccount/{model}/resolve/main/train_{i}/best_model/special_tokens_map.json -P {curr_dir}"
            )
            os.system(
                f"wget -q https://huggingface.co/researchaccount/{model}/resolve/main/train_{i}/best_model/tokenizer_config.json -P {curr_dir}"
            )
            os.system(
                f"wget -q https://huggingface.co/researchaccount/{model}/resolve/main/train_{i}/best_model/training_args.bin -P {curr_dir}"
            )
            os.system(
                f"wget -q https://huggingface.co/researchaccount/{model}/resolve/main/train_{i}/best_model/vocab.txt -P {curr_dir}"
            )


def softmax(x):
    return np.exp(x) / sum(np.exp(x))


def ga(file):
    code = """
    <!-- Global site tag (gtag.js) - Google Analytics -->
        <script async src="https://www.googletagmanager.com/gtag/js?id=G-NH9HWCW08F"></script>
        <script>
        window.dataLayer = window.dataLayer || [];
        function gtag(){dataLayer.push(arguments);}
        gtag('js', new Date());
        gtag('config', 'G-NH9HWCW08F');
    </script>
    """

    a = os.path.dirname(file) + "/static/index.html"
    with open(a, "r") as f:
        data = f.read()
        if len(re.findall("G-", data)) == 0:
            with open(a, "w") as ff:
                newdata = re.sub("<head>", "<head>" + code, data)
                ff.write(newdata)