Updates

app.py

@@ -1,16 +1,8 @@
-import os
-
-import gradio as gr
-
 import gradio as gr
 from find_similar_issues import get_similar_issues
 import requests
-from html2image import Html2Image
-import io
 
-hti = Html2Image(size=(1920, 1080 * 3))
-
-from defaults import OWNER, REPO, TOKEN
+from defaults import OWNER, REPO
 
 
 def get_query_issue_information(issue_no, token):
@@ -30,35 +22,17 @@ def get_query_issue_information(issue_no, token):
     return request.json()
 
 
-def find_similar_issues(issue, token):
-    similar_issues = get_similar_issues(issue, token=token)
-    similar_issues_summary = [f"#{issue['number']} - {issue['title']}" for issue in similar_issues]
-    return similar_issues_summary
-
-
-def render_issue_as_image(issue, filename="image.png"):
-    url = issue["html_url"]
-    print(url)
-    hti.screenshot(url=url, save_as=filename)
-    return filename
-
-
-def run_find_similar_issues(issue, token, n_issues):
-    issue_information = get_query_issue_information(issue, token=token)
-    # issue_information_summary = f"#{issue_information['number']} - {issue_information['title']}\n\n{issue_information['body']}"
-    similar_issues = get_similar_issues(issue, token=token, top_k=n_issues)
-    # similar_issues_summary = [f"#{issue['number']} - {issue['title']}" for issue in similar_issues]
-
-    issue_image = render_issue_as_image(issue_information, filename="query_issue.png")
-
-    image_names = []
-    for i, issue in enumerate(similar_issues):
-        image_names.append(render_issue_as_image(issue, filename=f"image{i}.png"))
+def run_find_similar_issues(token, n_issues, issue_no, query):
+    if issue_no == "":
+        issue_no = None
+    if query == "":
+        query = None
 
-    # return issue_information_summary, image_names
-    page_html = requests.get(issue_information["html_url"]).text
+    similar_issues = get_similar_issues(issue_no=issue_no, query=query, token=token, top_k=n_issues)
 
-    return issue_image, page_html, image_names
+    issues_html = [f"<a href='{issue['html_url']}' target='_blank'>#{issue['number']} - {issue['title']}</a>" for issue in similar_issues]
+    issues_html = "<br>".join(issues_html)
+    return issues_html
 
 
 with gr.Blocks(title="Github Bot") as demo:
@@ -66,29 +40,20 @@ with gr.Blocks(title="Github Bot") as demo:
         with gr.Row():
             with gr.Column():
                 with gr.Row():
-                    issue = gr.Textbox(label="Github Issue", placeholder="Github issue you want to find similar issues to")
-                    token = gr.Textbox(label="Github Token", placeholder="Your github token for authentication. This is not stored anywhere.")
+                    issue_no = gr.Textbox(label="Github Issue", placeholder="Github issue you want to find similar issues to")
+                    query = gr.Textbox(label="Query", placeholder="Search for issues")
                 with gr.Row():
-                    n_issues = gr.Slider(1, 50, value=5, label="Number of similar issues", info="Choose between 1 and 50")
+                    token = gr.Textbox(label="Github Token", placeholder="Your github token for authentication. This is not stored anywhere.")
+                    n_issues = gr.Slider(1, 50, value=5, step=1, label="Number of similar issues", info="Choose between 1 and 50")
 
         with gr.Row():
             submit_button = gr.Button(value="Submit")
 
         with gr.Row():
-            with gr.Column():
-                issue_image = gr.Image(type="filepath", label="Your issue")
-            with gr.Column():
-                similar_issues_screenshots = gr.Gallery(label="Similar Issues")
-                issue_text = gr.HTML(label="Issue text", elem_id="issue_text")
-                submit_button.click(run_find_similar_issues, outputs=[issue_image, issue_text, similar_issues_screenshots], inputs=[issue, token, n_issues])
-
-    with gr.Tab("Search issues"):
-        with gr.Row():
-            query = gr.Textbox(label="Query", placeholder="Search for issues")
-        with gr.Row():
-            token = gr.Textbox(label="Github Token", placeholder="Your github token for authentication. This is not stored anywhere.")
-        with gr.Row():
-            pass
+            with gr.Row():
+                issues_html = gr.HTML(label="Issue text", elem_id="issue_html")
+            with gr.Row():
+                submit_button.click(run_find_similar_issues, outputs=[issues_html], inputs=[token, n_issues, issue_no, query])
 
     with gr.Tab("Find maintainers to ping"):
         with gr.Row():

find_similar_issues.py

@@ -40,9 +40,10 @@ def cosine_similarity(a, b):
     return np.dot(a, b.T) / (np.linalg.norm(a, axis=1) * np.linalg.norm(b, axis=1))
 
 
-def get_similar_issues(issue_no, top_k=5, token=TOKEN, owner=OWNER, repo=REPO):
+
+def get_issue(issue_no, token=TOKEN, owner=OWNER, repo=REPO):
     """
-    Function to find similar issues
+    Function to get issue from GitHub
     """
     url = f"https://api.github.com/repos/{owner}/{repo}/issues"
     headers = {
@@ -55,11 +56,26 @@ def get_similar_issues(issue_no, top_k=5, token=TOKEN, owner=OWNER, repo=REPO):
         f"https://api.github.com/repos/{OWNER}/{REPO}/issues/{issue_no}",
         headers=headers,
     )
-
     if request.status_code != 200:
         raise ValueError(f"Request failed with status code {request.status_code}")
+    return request.json()
+
+
+def get_similar_issues(issue_no, query, top_k=5, token=TOKEN, owner=OWNER, repo=REPO):
+    """
+    Function to find similar issues
+    """
+    if issue_no is not None and query is not None:
+        raise ValueError("Only one of issue_no or query can be provided")
+
+    if issue_no is not None and query is not None:
+        raise ValueError("Only one of issue_no or query can be provided")
+
+    if issue_no is not None:
+        issue = get_issue(issue_no, token=token, owner=owner, repo=repo)
+        query = issue["title"] + "\n" +issue["body"]
 
-    query_embedding = model.encode(request.json()["body"])
+    query_embedding = model.encode(query)
     query_embedding = query_embedding.reshape(1, -1)
     embeddings = load_embeddings()
 
@@ -82,10 +98,11 @@ def get_similar_issues(issue_no, top_k=5, token=TOKEN, owner=OWNER, repo=REPO):
 
 if __name__ == "__main__":
     parser = argparse.ArgumentParser()
-    parser.add_argument("issue_no", type=int)
+    parser.add_argument("--issue_no", type=int, default=None)
+    parser.add_argument("--query", type=str, default=None)
     parser.add_argument("--top_k", type=int, default=5)
     parser.add_argument("--token", type=str, default=TOKEN)
     parser.add_argument("--owner", type=str, default=OWNER)
     parser.add_argument("--repo", type=str, default=REPO)
     args = parser.parse_args()
-    get_similar_issues(args.issue_no, args.top_k, args.token, args.owner, args.repo)
+    get_similar_issues(**vars(args))

get_issues.py

@@ -67,7 +67,15 @@ def get_issues(
     }
     last_issue_number = get_last_issue_number(file_path=output_filename)
     per_page = 100
-    page = last_issue_number // per_page + 1
+
+    if os.path.exists(output_filename):
+        with open(output_filename, "r") as f:
+            num_lines = sum(1 for line in f)
+    else:
+        num_lines = 0
+
+    # Get the number of pages
+    page = num_lines // per_page + 1
     query_params = {
         "state": "all",
         "per_page": per_page,

get_topic.py

@@ -1,4 +1,8 @@
-"text": {}
+
+import json
+
+with open("issues_dict.json", "r") as f:
+    issues = json.load(f)
 
 topic_maintainers_map ={
     "text models": ["@ArthurZucker", "@younesbelkada"],
@@ -30,14 +34,24 @@ topic_maintainers_map ={
 }
 
 
+issue_no = 2781
+issue = issues[str(issue_no)]
+
+
 from transformers import AutoTokenizer, LlamaForCausalLM
 
 model = LlamaForCausalLM.from_pretrained("meta-llama/Llama-2-7b-hf")
 tokenizer = AutoTokenizer.from_pretrained("meta-llama/Llama-2-7b-hf")
 
-prompt = f"Which of the following topics {list(topic_maintainers_map.keys())} is this issue about:\n{issue['body']}"
+# prompt = f"Which of the following topics {list(topic_maintainers_map.keys())} is this issue about:\n{issue['body']}"
+prompt = f"What is the provided issue about? Pick up to 3 topics from the following list: {list(topic_maintainers_map.keys())} \nIssue:\n{issue['body']}"
 inputs = tokenizer(prompt, return_tensors="pt")
 
+prefix_len = inputs.input_ids.shape[1]
+
 # Generate
-generate_ids = model.generate(inputs.input_ids, max_length=30)
-tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
+generate_ids = model.generate(inputs.input_ids, max_length=30 + prefix_len)
+outputs = tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
+print(outputs[prefix_len:])
+
+print("TITLE", issue["number"] + " " + issue["title"])

update_embeddings.py

@@ -94,7 +94,6 @@ def embed_issues(
                 embeddings[index] = embedding
             else:
                 index = len(embeddings)
-                # embeddings = np.concatenate([embeddings, embedding.reshape(1, -1)])
                 embeddings.append(embedding)
                 issue_to_embedding_index[issue_id] = index
                 embedding_to_issue_index[index] = issue_id

update_stored_issues.py

@@ -70,8 +70,7 @@ def get_issues(
     page = 1
     query_params = {
         "state": "all",
-        "since": "2024-02-01T11:33:35Z",
-        # "since": most_recent,
+        "since": most_recent,
         "sort": "created",
         "direction": "asc",
         "page": page,
@@ -110,6 +109,9 @@ def get_issues(
 
     issue_lines_map = {issue["number"]: issue for issue in new_lines}
 
+    updated_issues = []
+
+    # Update any issues that already exist
     with open(input_filename, "r") as f:
         with open("tmp_" + output_filename, "a") as g:
             for line in f:
@@ -118,11 +120,21 @@ def get_issues(
                 if number in issue_lines_map:
                     g.write(json.dumps(issue_lines_map[number]))
                     g.write("\n")
+                    updated_issues.append(number)
                 else:
                     g.write(line)
 
+    # Append any new issues
+    new_issues = [issue for issue in new_lines if issue["number"] not in updated_issues]
+    with open("tmp_" + output_filename, "a") as g:
+        for issue in new_issues:
+            g.write(json.dumps(issue))
+            g.write("\n")
+
+    # Overwrite the old file with the new file
     os.rename("tmp_" + output_filename, output_filename)
 
+    # Save a record of the updated issues for the embedding update
     with open("updated_issues.json", "w") as f:
         json.dump(issue_lines_map, f, indent=4, sort_keys=True)