import json
import logging
import os
import re
import string
import gradio as gr
from elasticsearch import Elasticsearch
from elasticsearch_dsl import Search, Q
es = Elasticsearch(os.environ.get("host"), timeout=10)
def mark_tokens_bold(string, tokens):
for token in tokens:
pattern = re.escape(token) #r"\b" + re.escape(token) + r"\b"
string = re.sub(pattern, "" + token + "", string)
return string
def process_results(results):
if len(results) == 0:
return """
No results retrieved.
Repository: {}
StarCoder: Dataset Search 🔍
When you use StarCoder to generate code it might produce exact copies of code in the pretraining dataset. In that case, the code license might have requirements to comply with. With this search tool we aim to provide help to find out where the code came from, in order for the user to comply with licensing requirements in case the code produced by StarCoder belongs to an already existing repository. For exact matches, enclose your query in double quotes.""" if __name__ == "__main__": demo = gr.Blocks( css=".gradio-container {background-color: #20233fff; color:white}" ) with demo: with gr.Row(): gr.Markdown(value=description) with gr.Row(): query = gr.Textbox(lines=5, placeholder="Type your query here...", label="Query") with gr.Row(): k = gr.Slider(1, 100, value=10, step=1, label="Max Results") with gr.Row(): submit_btn = gr.Button("Submit") with gr.Row(): results = gr.HTML(label="Results", value="