Hugging Face's logo

Spaces:
PhilippSpohn
/
tokenprob
Running

App Files Community
tokenprob
File size: 2,457 Bytes 
b9b96cc
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3bd5de9
 
 
 
b9b96cc
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
 
<!DOCTYPE html>
<html>
<head>
    <title>LLM Token Visualization</title>
    <link rel="stylesheet" href="{{ url_for('static', filename='style.css') }}">
    <script src="https://unpkg.com/@popperjs/core@2"></script>
    <script src="https://unpkg.com/tippy.js@6"></script>
    <link rel="stylesheet" href="https://unpkg.com/tippy.js@6/themes/light.css">
    <link href="https://fonts.googleapis.com/css2?family=Inter:wght@400;500;600&display=swap" rel="stylesheet">
</head>
<body>
    <div class="container">
        <header>
            <h1>LLM Token Visualization</h1>
            <p class="subtitle">Analyze how language models process and predict text</p>
        </header>

        <div class="control-panel">
            <div class="input-group">
                <label for="model-select">Model:</label>
                <select id="model-select" class="styled-select">
                    {% for model in models %}
                        <option value="{{ model }}">{{ model }}</option>
                    {% endfor %}
                </select>
            </div>

            <div class="input-group">
                <label for="input-text">Text to Analyze:</label>
                <textarea id="input-text" placeholder="Enter your text here..."></textarea>
            </div>

            <button id="analyze-button" class="primary-button">
                <span class="button-text">Analyze</span>
                <div class="button-spinner hidden"></div>
            </button>
        </div>

        <div id="output" class="output-panel">
            <div class="output-section">
                <h2>Token Analysis</h2>
                <div id="colored-text" class="token-display"></div>
            </div>

            <div class="stats-section">
                <h2>Statistics</h2>
                <div class="stats-grid">
                    <div class="stat-card">
                        <div class="stat-label">Joint Log-Likelihood</div>
                        <div class="stat-value" id="joint-log-likelihood">-</div>
                    </div>
                    <div class="stat-card">
                        <div class="stat-label">Average Log-Likelihood</div>
                        <div class="stat-value" id="average-log-likelihood">-</div>
                    </div>
                </div>
            </div>
        </div>
    </div>

    <script src="{{ url_for('static', filename='script.js') }}"></script>
</body>
</html>