flask work

app.py

@@ -1,58 +1,105 @@
+import os
+import random
+import glob
 import json
 
-import gradio as gr
+import numpy as np
+from flask import Flask, render_template, request
+
+app = Flask(__name__)
 
 
 with open("problems.json") as f:
     problems = json.load(f)
     problem_choices = [q["question_title"] for q in problems]
 
+random_idxs = list(range(len(problems)))
+random.shuffle(random_idxs)
+
 with open("all_outputs.json") as f:
     all_outputs = json.load(f)
-    model_choices = list(all_outputs.keys())
-
-
-def update_view(problem_index, model_name):
-    code = all_outputs[model_name][problem_index]["code_list"][0]
-    correct = all_outputs[model_name][problem_index]["pass1_list"][0]
-    code_viewer = gr.Code(
-        label=f"Code for Model {model_name} on {problems[problem_index]['question_title']} problem. Passed: {correct}",
-        language="python",
-        lines=10,
-        value=code,
-        interactive=False,
-    )
-    return code_viewer
+    all_models = list(all_outputs.keys())
 
 
-with gr.Blocks() as demo:
-    ## dropdown
-    problem_dropdown = gr.Dropdown(
-        label="Selected Problem",
-        choices=problem_choices,
-        value=problem_choices[0],
-        type="index",
-    )
+num_questions_filtered = len(problems)
 
-    model_dropdown = gr.Dropdown(
-        label="Selected Model",
-        choices=model_choices,
-        value="GPT-4-Turbo-1106",
-        type="value",
-    )
+all_correctness_by_problem = {
+    idx: {model: np.mean(all_outputs[model][idx]["pass1_list"]) for model in all_models}
+    for idx in random_idxs
+}
+
+
+def calculate_color(performance):
+    # Convert performance to a value between 0 and 1
+    # Calculate the red and green components of the color
+    if performance > 0.75:
+        return f"rgba(0, 150, 0, 0.5)"
+    elif performance > 0.5:
+        return f"rgba(50, 150, 0, {performance})"
+    elif performance > 0.25:
+        return f"rgba(150, 50, 0, {1-performance})"
+    else:
+        return f"rgba(150, 0, 0,  0.5)"
 
-    code_viewer = update_view(0, model_choices[0])
 
-    problem_dropdown.change(
-        update_view, inputs=[problem_dropdown, model_dropdown], outputs=[code_viewer]
+all_evaluations_by_problem_colored = [
+    (
+        trueidx,
+        {
+            model: {
+                "correctness": f"{all_correctness_by_problem[idx][model]*100:.1f}",
+                "correctness_color": calculate_color(
+                    all_correctness_by_problem[idx][model]
+                ),
+            }
+            for model in all_models
+        },
+        problems[idx]["difficulty"],
     )
+    for trueidx, idx in enumerate(random_idxs)
+]
 
-    model_dropdown.change(
-        update_view, inputs=[problem_dropdown, model_dropdown], outputs=[code_viewer]
+all_data_for_view_formatted = {
+    model: [
+        [{"code": a, "pass1": b} for a, b in zip(row["code_list"], row["pass1_list"])]
+        # print(row)
+        for idx in random_idxs
+        for row in [resp[idx]]
+    ]
+    for model, resp in all_outputs.items()
+}
+
+
+@app.route("/")
+def home():
+    # Fetch your data here
+    print(all_models)
+    return render_template(
+        "index.html", models=all_models, problems=all_evaluations_by_problem_colored
     )
 
-    ## code viewer
+
+@app.route("/problem/<int:problem_idx>")
+def problem(problem_idx):
+    # Fetch your data here
+
+    data = {
+        model: all_data_for_view_formatted[model][problem_idx] for model in all_models
+    }
+    evaluation = all_evaluations_by_problem_colored[problem_idx][1]
+    question = problems[problem_idx]
+
+    # print(data)
+
+    return render_template(
+        "problem.html",
+        problem_idx=problem_idx,
+        evaluation=evaluation,
+        models=all_models,
+        question=question,
+        data=data,
+    )
 
 
 if __name__ == "__main__":
-    demo.launch()
+    app.run()

problems.json

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5328fbef3d2dd4d104bf71c3d0a0d1d9380756049b71ded66dc65b5cdc75aac2
-size 855054
+oid sha256:787405a4cc8aba30c50f7e0f3bd2f2026f9035e41471400167ec1c27e22b7097
+size 870796

requirements.txt

@@ -0,0 +1,2 @@
+flask
+numpy

templates/index.html

@@ -0,0 +1,121 @@
+<!DOCTYPE html>
+<html>
+
+<head>
+    <style>
+        body {
+            font-family: Arial, sans-serif;
+            margin: 0;
+            padding: 20px;
+            background-color: #f4f4f4;
+        }
+
+        #checkboxes {
+            margin-bottom: 20px;
+        }
+
+        .model-checkbox+label {
+            margin-right: 10px;
+            padding: 5px 10px;
+            background-color: #e7e7e7;
+            border-radius: 5px;
+            cursor: pointer;
+            user-select: none;
+        }
+
+        .model-checkbox {
+            display: none;
+            /* Hide the default checkbox */
+        }
+
+        .model-checkbox:checked+label {
+            background-color: #d0e6a5;
+            color: #0b4d03;
+        }
+
+        table {
+            width: 100%;
+            border-collapse: collapse;
+            background-color: #fff;
+            box-shadow: 0 2px 4px rgba(0, 0, 0, 0.1);
+            margin: auto;
+            overflow: hidden;
+            /* Add this line to handle overflow */
+        }
+
+        th,
+        td {
+            border: 1px solid #ddd;
+            padding: 10px 15px;
+            text-align: center;
+        }
+
+        th {
+            background-color: #f8f8f8;
+            font-weight: bold;
+        }
+
+        tr:nth-child(even) {
+            background-color: #f2f2f2;
+        }
+
+        .performance-bar {
+            display: block;
+            height: 20px;
+            background: linear-gradient(to right, #00ff00 0%, #ff0000 100%);
+            border-radius: 5px;
+        }
+
+        .performance {
+            font-weight: bold;
+            color: #fff;
+            text-align: center;
+            line-height: 20px;
+            height: 20px;
+            margin: 0;
+        }
+    </style>
+
+</head>
+
+<body>
+
+
+
+    <table id='model-table' style="align:center">
+        <thead>
+            <tr>
+                <th>Problem ID</th>
+                <th>Difficulty</th>
+                {% for model in models %}
+                <th class="column-{{ model }}">{{ model }}</th>
+                {% endfor %}
+            </tr>
+        </thead>
+        <tbody>
+            {% for problem_idx, problem, difficulty in problems %}
+            <tr>
+                <td> <a href="{{ url_for('problem', problem_idx=problem_idx) }}"> {{ problem_idx }} </a> </td>
+                <td> {{ difficulty }} </td>
+                {% for model in models %}
+                <td style="background-color: {{ problem[model]['correctness_color'] }};" class="column-{{ model }}">
+                    {% if model in problem %}
+                    <div>
+                        <!-- {{ problem[model]['correctness'] }}% -->
+                        {{ problem[model]['correctness'] }}%
+                    </div>
+                    {% else %}
+                    -
+                    {% endif %}
+                </td>
+                {% endfor %}
+            </tr>
+            {% endfor %}
+
+        </tbody>
+    </table>
+
+
+</body>
+
+</html>

templates/problem.html

@@ -0,0 +1,325 @@
+<!DOCTYPE html>
+<html>
+
+<head>
+    <style>
+        pre {
+            white-space: pre-wrap;
+            /* Wraps the text */
+            word-break: break-word;
+            /* Ensures words break to prevent overflow */
+            background-color: #f4f4f4;
+            /* Light grey background */
+            padding: 2px;
+            /* Padding around the text */
+            border-radius: 2px;
+            /* Rounded corners */
+            border: 1px solid #ccc;
+            /* Light grey border */
+        }
+
+        .container {
+            display: flex;
+            /* Use flexbox to layout children */
+            justify-content: space-between;
+            /* Space between the children */
+            margin-bottom: 10px;
+            /* Space between each container */
+            background-color: #ddd;
+            /* Debug: background color */
+        }
+
+        .sub-container {
+            flex: 1;
+            /* Each sub-container takes equal width */
+            padding: 20px;
+            /* Padding inside each sub-container */
+            background-color: #f4f4f4;
+            /* Background color */
+            border-radius: 2px;
+            /* Rounded corners */
+            border: 1px solid #ccc;
+            /* Border */
+            margin: 0 10px;
+            /* Margin between sub-containers */
+        }
+
+        body {
+            font-family: Arial, sans-serif;
+            margin: 0;
+            padding: 20px;
+            background-color: #f4f4f4;
+        }
+
+        #checkboxes {
+            margin-bottom: 20px;
+        }
+
+        .model-checkbox+label {
+            margin-right: 10px;
+            padding: 5px 10px;
+            background-color: #e7e7e7;
+            border-radius: 5px;
+            cursor: pointer;
+            user-select: none;
+        }
+
+        .model-checkbox {
+            display: none;
+            /* Hide the default checkbox */
+        }
+
+        .model-checkbox:checked+label {
+            background-color: #d0e6a5;
+            color: #0b4d03;
+        }
+
+        .model {
+            border: 1px solid black;
+            padding: 10px;
+        }
+
+        .counter {
+            margin-top: 20px;
+        }
+
+        .passed code.hljs {
+            background-color: #e6ffe6 !important;
+            /* light green */
+            color: black !important;
+        }
+
+        .failed code.hljs {
+            background-color: #ffe6e6 !important;
+            /* light red */
+            color: black !important;
+        }
+
+        .solution {
+            display: none;
+            /* Hide all solutions by default */
+        }
+
+        .solution.active {
+            display: block;
+            /* Only show the active solution */
+        }
+
+        .button-container {
+            text-align: center;
+            /* Center the button container */
+            margin-top: 20px;
+            /* Add some space above the button */
+        }
+
+        .other-button {
+            display: inline-block;
+            padding: 10px 20px;
+            background-color: #BBBBBB;
+            color: white;
+            text-decoration: none;
+            border-radius: 5px;
+            font-weight: bold;
+        }
+
+        .other-button:hover {
+            background-color: #0056b3;
+        }
+    </style>
+    <script src="https://code.jquery.com/jquery-3.5.1.min.js"></script>
+    <link rel="stylesheet" href="//cdnjs.cloudflare.com/ajax/libs/highlight.js/10.7.2/styles/default.min.css">
+
+    <script>
+        // Wait for the DOM to be fully loaded
+        document.addEventListener('DOMContentLoaded', function () {
+            // Get all checkboxes with the class 'model-checkbox'
+            var checkboxes = document.querySelectorAll('.model-checkbox');
+
+            // Load the checkbox states from localStorage and apply them
+            checkboxes.forEach(function (checkbox) {
+                var model = checkbox.id.replace('checkbox-', '');
+                var savedState = localStorage.getItem('checkbox-' + model);
+                if (savedState !== null) {
+                    checkbox.checked = savedState === 'true';
+                    var modelElement = document.getElementById('code-' + model);
+                    modelElement.style.display = checkbox.checked ? 'block' : 'none';
+                }
+            });
+
+
+
+            // Add a change event listener to each checkbox
+            checkboxes.forEach(function (checkbox) {
+                checkbox.addEventListener('change', function () {
+                    // Get the model name from the checkbox ID
+                    var model = this.id.replace('checkbox-', '');
+
+                    // Get the model element
+                    var modelElement = document.getElementById('code-' + model);
+
+                    // toggle the display of the model element
+                    modelElement.style.display = this.checked ? 'block' : 'none';
+
+                    // Save the state of all checkboxes to localStorage
+                    checkboxes.forEach(function (cb) {
+                        var modelId = cb.id.replace('checkbox-', '');
+                        localStorage.setItem('checkbox-' + modelId, cb.checked);
+                    });
+
+                });
+            });
+        });
+    </script>
+</head>
+
+<body>
+    <h1>Problem {{ problem_idx }}</h1>
+
+
+    <div class="button-container">
+        {% set next_problem_idx = problem_idx + 1 %}
+        <a href="{{ url_for('problem', problem_idx=next_problem_idx) }}" class="other-button">Next Problem</a>
+
+        {% set prev_problem_idx = problem_idx - 1 %}
+        <a href="{{ url_for('problem', problem_idx=prev_problem_idx) }}" class="other-button">Prev Problem</a>
+    </div>
+
+    <br />
+    <br />
+
+    <!-- <div id="checkboxes">
+        {% for model in models %}
+        <input type="checkbox" class="model-checkbox" id="checkbox-{{ model }}" checked>
+        <label for="checkbox-{{ model }}">{{ model }}@{{ evaluation[model]['correctness'] }}/{{
+            evaluation[model]['performance'] }}</label>
+        {% endfor %}
+    </div> -->
+
+    <div class="container">
+        <div class="sub-container">
+            <h3>{{ question['question_title'] }} || {{ question['difficulty'] }} || {{
+                question['contest_date'].split('T')[0]}}</h3>
+
+            <pre overflow="auto">
+{{ question['question_content'] }}
+            </pre>
+        </div>
+
+        <div class="sub-container">
+            <select id="model-select" class="model-select">
+                {% for model in models %}
+                <option value="{{ model }}">{{ model }}@{{ evaluation[model]['correctness']
+                    }}/{{evaluation[model]['performance'] }}</option>
+                {% endfor %}
+            </select>
+
+
+            <div id="all-results">
+
+                {% for model in models %}
+
+                <div class="model" id="code-{{ model }}" style="display: none;">
+                    <h2> {{ model }}</h2>
+
+                    <div class="solutions-container">
+                        {% for code in data[model] %}
+
+                        <div class="solution{% if loop.first %} active{% endif %}" id="solution-{{ loop.index }}">
+
+                            <ul>
+                                <li> solution idx -- {{ loop.index }} </li>
+                                <li>correctness -- {{ code['pass1'] }}</li>
+                            </ul>
+
+                            <div>
+                                <pre
+                                    class="{{ 'passed' if code['correctness'] else 'failed' }}"><code class="language-python">{{ code['code'] }}</code></pre>
+                            </div>
+                        </div>
+
+                        {% endfor %}
+                    </div>
+                    <button data-model="{{ model }}" class="prev-solution">Previous</button>
+                    <button data-model="{{ model }}" class="next-solution">Next</button>
+
+                </div>
+                {% endfor %}
+            </div>
+        </div>
+    </div>
+    </div>
+
+
+
+
+
+    <script src="//cdnjs.cloudflare.com/ajax/libs/highlight.js/10.7.2/highlight.min.js"></script>
+    <script>hljs.highlightAll();</script>
+
+    <script>
+        document.addEventListener('DOMContentLoaded', function () {
+            var allModels = document.querySelectorAll('.model');
+            console.log(allModels);
+
+            allModels.forEach(function (model) {
+                console.log(model);
+                var solutions = model.querySelectorAll('.solution');
+                var totalSolutions = solutions.length;
+                var currentIndex = 0;
+                console.log(totalSolutions, currentIndex);
+
+                function updateActiveSolution(index) {
+                    solutions.forEach(function (solution, i) {
+                        if (i === index) {
+                            solution.classList.add('active');
+                        } else {
+                            solution.classList.remove('active');
+                        }
+                    });
+                }
+
+                model.querySelector('.prev-solution').addEventListener('click', function () {
+                    console.log(currentIndex, model);
+                    currentIndex = (currentIndex - 1 + totalSolutions) % totalSolutions;
+                    updateActiveSolution(currentIndex);
+                });
+
+                model.querySelector('.next-solution').addEventListener('click', function () {
+                    console.log(currentIndex, model);
+                    currentIndex = (currentIndex + 1) % totalSolutions;
+                    updateActiveSolution(currentIndex);
+                });
+            });
+        });
+    </script>
+
+    <script>
+        document.addEventListener('DOMContentLoaded', function () {
+            var modelSelect = document.getElementById('model-select');
+            var models = document.querySelectorAll('.model');
+            modelSelect.addEventListener('change', function () {
+                var selectedModel = this.value;
+                models.forEach(function (model) {
+                    if (model.id === 'code-' + selectedModel) {
+                        model.style.display = 'block';
+                    } else {
+                        model.style.display = 'none';
+                    }
+                });
+            });
+        });
+        document.addEventListener('DOMContentLoaded', function () {
+            var selectElement = document.getElementById('model-select');
+            selectElement.value = 'GPT-4-Turbo-2024-04-09';
+
+            var codeElement = document.getElementById('code-GPT-4-Turbo-2024-04-09');
+            if (codeElement) {
+                codeElement.style.display = 'block';
+            }
+        });
+
+    </script>
+
+</body>
+
+</html>